﻿using System;
using System.Collections.Generic;
using System.Configuration;
using System.Data;
using System.Data.SqlClient;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Text.RegularExpressions;
using DiscoveryLogic.Analytics.DataObjects;
using DiscoveryLogic.Analytics.DataObjects.Networks;
using DiscoveryLogic.Analytics.DataObjects.Rules;
using DiscoveryLogic.Common.Utilities;
using DiscoveryLogic.Common.Utilities.TextMatch;
using DiscoveryLogic.Workflows.Components.Entities;
using DiscoveryLogic.Workflows.Components.Entities.Distances.Comparers;

namespace DiscoveryLogic.Analytics.BusinessLogic.Clusters
{
    public class EntityClusterWorker:IDisposable
    {
        private EntityClusterRule _Rule;
        private readonly Regex _Splitter;
        private readonly List<string> _StopWords;
        private DictionaryUtil _Dictionary = null;

        public EntityClusterWorker(EntityClusterRule rule)
        {
            this._Rule = rule;
            this._Splitter = new Regex("([ \\t{}():;. \n])");
            StopWordsUtil stopWordsUtil=new StopWordsUtil(StopWordsUtil.StopWordList.PubmedStopWordList);
            this._StopWords = stopWordsUtil.StopWords;
            if (ConfigurationManager.AppSettings["WordNetConnection"] != null)
            {
                string connStr = ConfigurationManager.AppSettings.Get("WordNetConnection");
                this._Dictionary=new DictionaryUtil(connStr,"Word","Lemma");
            }
        }

        public void ClusterEntities(List<EntityBase> entities, 
            ref List<NetworkCluster> networkClusters)
        {
            networkClusters = new List<NetworkCluster>();
            Type objType = EntityTypeUtil.GetType(this._Rule.SrcEntityType);
            Dictionary<string, double> termWeights = new Dictionary<string, double>();
            Dictionary<EntityBase, Dictionary<string, List<TermFreq>>> indexedEntityFields =
                this.IndexEntityContents(objType, entities, ref termWeights);
            if (termWeights.Count > 250)
            {
                List<TermWeight> tws = this.ToTermWeightList(termWeights);
                tws.Sort();
                List<TermWeight> clone=new List<TermWeight>();
                for(int i=0;i<250;i++)
                {
                    clone.Add(tws[i]);
                }
                tws = clone;
                termWeights = this.ToTermWeightDictionary(tws);
            }
            // partition
            Dictionary<int,List<EntityBase>> partitions=new Dictionary<int, List<EntityBase>>();
            if(this._Rule.AssocAttrWeights !=null && this._Rule.AssocAttrWeights.Count>0)
            {
                throw new NotImplementedException("Associative attribute used in partition is not implemented");
            }
            else
            {
                int partitionID = 1;
                List<EntityBase> entities1 =new List<EntityBase>();
                foreach(EntityBase entity in entities)
                {
                    if(entity.EntityType==this._Rule.SrcEntityType)
                        entities1.Add(entity);
                }
                partitions.Add(partitionID, entities1);
            }

            int clusterID = 0;
            foreach(int partitionID in partitions.Keys)
            {
                List<EntityBase> entities1 = partitions[partitionID];
                if(entities1.Count>1)
                {
                    Dictionary<EntityBase, Dictionary<EntityBase, double>> scoreMatrix =
                        this.DoPairwiseComparison(indexedEntityFields, entities1);
                    Dictionary<EntityBase, int> clusters=new Dictionary<EntityBase, int>();
                    foreach(EntityBase fromEntity in scoreMatrix.Keys)
                    {
                        foreach(EntityBase toEntity in scoreMatrix[fromEntity].Keys)
                        {
                            double score = scoreMatrix[fromEntity][toEntity];
                            if(score >=this._Rule.ClusterThreshold)
                            {
                                if(clusters.ContainsKey(fromEntity) && !clusters.ContainsKey(toEntity))
                                {
                                    int existingClusterID = clusters[fromEntity];
                                    clusters.Add(toEntity,existingClusterID);
                                }
                                else if(clusters.ContainsKey(toEntity) && !clusters.ContainsKey(fromEntity))
                                {
                                    int existingClusterID = clusters[toEntity];
                                    clusters.Add(fromEntity, existingClusterID);
                                }
                                else if(!clusters.ContainsKey(fromEntity) && !clusters.ContainsKey(toEntity))
                                {
                                    clusterID++;
                                    clusters.Add(fromEntity,clusterID);
                                    if (toEntity != fromEntity)
                                        clusters.Add(toEntity, clusterID);
                                }
                            }
                        }
                    }
                    List<int> newClusterIDs=new List<int>();
                    foreach(int newClusterID in clusters.Values)
                    {
                        if(!newClusterIDs.Contains(newClusterID))
                            newClusterIDs.Add(newClusterID);
                    }
                    foreach(int newClusterID in newClusterIDs)
                    {
                        NetworkCluster networkCluster =
                            new NetworkCluster(newClusterID, 0, newClusterID.ToString());
                        List<EntityBase> entities2=new List<EntityBase>();
                        foreach(EntityBase entity in clusters.Keys)
                        {
                            if(clusters[entity]==newClusterID)
                                entities2.Add(entity);
                        }
                        networkClusters.Add(networkCluster);
                    }
                }
                else
                {
                    clusterID++;
                    NetworkCluster networkCluster=new NetworkCluster(
                        clusterID, 0, clusterID.ToString());
                    networkClusters.Add(networkCluster);
                }
            }
        }

        public Dictionary<int,List<ClusterAssociation>> DoFuzzyCluster()
        {
            Dictionary<int, List<ClusterAssociation>> clusterAssocs=new Dictionary<int, List<ClusterAssociation>>();

            return clusterAssocs;
        }

        private Dictionary<EntityBase,Dictionary<EntityBase,double>> DoPairwiseComparison(
            Dictionary<EntityBase, Dictionary<string,List<TermFreq>>> indexedEntityFields,
            List<EntityBase> entities)
        {
            Dictionary<EntityBase,Dictionary<EntityBase, double>> scoreMatrix=new Dictionary<EntityBase, Dictionary<EntityBase, double>>();
            for(int i=0;i<entities.Count;i++)
            {
                EntityBase fromEntity = entities[i];
                Dictionary<EntityBase, double> toEntityScores=new Dictionary<EntityBase, double>();
                for(int k=i+1;k<entities.Count;k++)
                {
                    EntityBase toEntity = entities[k];
                    double score = 0;
                    foreach(string attrName in this._Rule.DescAttrWeights.Keys)
                    {
                        IAttributeComparer comparer = this._Rule.Comparers[attrName];
                        object val1 = null;
                        if (indexedEntityFields.ContainsKey(fromEntity) && indexedEntityFields[fromEntity].ContainsKey(attrName))
                            val1 = indexedEntityFields[fromEntity][attrName];
                        object val2 = null;
                        if(indexedEntityFields.ContainsKey(toEntity) && indexedEntityFields[toEntity].ContainsKey(attrName))
                            val2 = indexedEntityFields[toEntity][attrName];
                        double penalty = 0;
                        if (this._Rule.Penalties != null && this._Rule.Penalties.ContainsKey(attrName))
                            penalty = this._Rule.Penalties[attrName];
                        double similarity = comparer.Compare(val1, val2, penalty, null, null);
                        score += similarity;
                    }
                    if (score >= this._Rule.PartitionThreshold)
                    {
                        foreach (string attrName in this._Rule.AssocAttrWeights.Keys)
                        {
                            IAttributeComparer comparer = this._Rule.Comparers[attrName];
                            PropertyInfo prop = entities[i].GetType().GetProperty(attrName);
                            object val1 = prop.GetValue(entities[i], null);
                            object val2 = prop.GetValue(entities[k], null);
                            double penalty = 0;
                            if (this._Rule.Penalties != null && this._Rule.Penalties.ContainsKey(attrName))
                                penalty = this._Rule.Penalties[attrName];
                            double similarity = comparer.Compare(val1, val2, penalty, null, null);
                            score += similarity;
                        }
                    }
                    toEntityScores.Add(entities[k], score);
                }
                scoreMatrix.Add(entities[i], toEntityScores);
            }

            for(int i=0;i<entities.Count;i++)
            {
                EntityBase fromEntity = entities[i];
                Dictionary<EntityBase, double> toEntityScores = scoreMatrix[fromEntity];
                for(int k=0;k<=i;k++)
                {
                    EntityBase toEntity = entities[k];
                    if(!toEntityScores.ContainsKey(toEntity))
                    {
                        if(i==k)
                            toEntityScores.Add(toEntity, 0);
                        else if(scoreMatrix.ContainsKey(toEntity) && scoreMatrix[toEntity].ContainsKey(fromEntity))
                            toEntityScores.Add(toEntity, scoreMatrix[toEntity][fromEntity]);
                    }
                }
                scoreMatrix[fromEntity] = toEntityScores;
            }
            return scoreMatrix;
        }

        #region index
        private Dictionary<EntityBase, Dictionary<string, List<TermFreq>>> IndexEntityContents(
            Type objType, List<EntityBase> entities,
            ref Dictionary<string, double> termWeights)
        {
            termWeights = new Dictionary<string, double>();
            Dictionary<string, List<TermFreq>> termFreqsByDoc = new Dictionary<string, List<TermFreq>>();
            Dictionary<EntityBase,Dictionary<string,List<TermFreq>>> indexedEntityFields =
                new Dictionary<EntityBase, Dictionary<string, List<TermFreq>>>();
            List<string> attrNames=new List<string>();
            if (this._Rule.DescAttrWeights != null && this._Rule.DescAttrWeights.Count > 0)
            {
                foreach (string attrName in this._Rule.DescAttrWeights.Keys)
                {
                    attrNames.Add(attrName);
                }
            }
            if (this._Rule.AssocAttrWeights != null && this._Rule.AssocAttrWeights.Count > 0)
            {
                foreach (string attrName in this._Rule.AssocAttrWeights.Keys)
                {
                    attrNames.Add(attrName);
                }
            }
            foreach (string attrName in attrNames)
            {
                PropertyInfo prop = objType.GetProperty(attrName);
                Dictionary<EntityBase, List<TermFreq>> indexedEntityField =
                    this.IndexEntityFieldContents(prop, entities);
                foreach (EntityBase entity in indexedEntityField.Keys)
                {
                    List<TermFreq> termFreqs = indexedEntityField[entity];
                    if(termFreqs==null)
                        continue;
                    termFreqs.Sort();
                    if (termFreqs.Count > 250)
                    {
                        List<TermFreq> clone = new List<TermFreq>();
                        for (int i = 0; i < 250; i++)
                            clone.Add(termFreqs[i]);
                        termFreqs = clone;
                    }
                    if (termFreqsByDoc.ContainsKey(entity.PKValue()))
                    {
                        List<TermFreq> existingTermFreqs = termFreqsByDoc[entity.PKValue()];
                        Dictionary<string, int> existTermFreqs2 = this.ToTermFreqDictionary(existingTermFreqs);
                        foreach (TermFreq tf in termFreqs)
                        {
                            if (existTermFreqs2.ContainsKey(tf.Term))
                                existTermFreqs2[tf.Term] = existTermFreqs2[tf.Term] + tf.Freq;
                            else
                                existTermFreqs2.Add(tf.Term, tf.Freq);
                        }
                        termFreqsByDoc[entity.PKValue()] = this.ToTermFreqList(existTermFreqs2);
                    }
                    else
                        termFreqsByDoc.Add(entity.PKValue(), termFreqs);
                    if (indexedEntityFields.ContainsKey(entity))
                    {
                        Dictionary<string, List<TermFreq>> indexedField = indexedEntityFields[entity];
                        indexedField.Add(attrName, termFreqs);
                        indexedEntityFields[entity] = indexedField;
                    }
                    else
                    {
                        Dictionary<string, List<TermFreq>> indexedField = new Dictionary<string, List<TermFreq>>();
                        indexedField.Add(attrName, termFreqs);
                        indexedEntityFields.Add(entity, indexedField);
                    }
                }
            }

            termWeights = this.CalculateTfidfScore(termFreqsByDoc);
            
            return indexedEntityFields;
        }

        private Dictionary<string,int> ToTermFreqDictionary(List<TermFreq> termFreqs)
        {
            Dictionary<string,int> termFrequencies=new Dictionary<string, int>();
            foreach(TermFreq termFreq in termFreqs)
            {
                termFrequencies.Add(termFreq.Term, termFreq.Freq);
            }
            return termFrequencies;
        }

        private List<TermFreq> ToTermFreqList(Dictionary<string,int> termFrequencies)
        {
            List<TermFreq> termFreqs=new List<TermFreq>();
            foreach(string term in termFrequencies.Keys)
            {
                TermFreq tf=new TermFreq();
                tf.Term = term;
                tf.Freq = termFrequencies[term];
                termFreqs.Add(tf);
            }
            return termFreqs;
        }

        private List<TermWeight> ToTermWeightList(Dictionary<string,double> tws)
        {
            List<TermWeight> termWeights=new List<TermWeight>();
            foreach(string term in tws.Keys)
            {
                TermWeight tw = new TermWeight();
                tw.Term = term;
                tw.Weight = tws[term];
                termWeights.Add(tw);
            }
            return termWeights;
        }

        private Dictionary<string, double> ToTermWeightDictionary(List<TermWeight> tws)
        {
            Dictionary<string, double> termWeights=new Dictionary<string, double>();
            foreach(TermWeight tw in tws)
            {
                termWeights.Add(tw.Term, tw.Weight);
            }
            return termWeights;
        }

        private Dictionary<EntityBase, List<TermFreq>> IndexEntityFieldContents(
            PropertyInfo textFieldProp, List<EntityBase> entities)
        {
            Dictionary<EntityBase, List<TermFreq>> indexedEntityFieldContent =
                new Dictionary<EntityBase, List<TermFreq>>();
            foreach(EntityBase entity in entities)
            {
                object fieldValue = textFieldProp.GetValue(entity, null);
                if(fieldValue !=null)
                {
                    string content = fieldValue.ToString();
                    List<TermFreq> termFreqs = this.Split(content);
                    indexedEntityFieldContent.Add(entity, termFreqs);
                }
            }
            return indexedEntityFieldContent;
        }

        private List<TermFreq> Split(string content)
        {
            if (string.IsNullOrEmpty(content))
                return null;
            List<TermFreq> termFreqs1 = new List<TermFreq>();
            Dictionary<string, int> termFreqs = new Dictionary<string, int>();
            content = content.ToLower().Trim();
            string[] words = this._Splitter.Split(content);
            try
            {
                foreach (string word in words)
                {
                    string word1 = StringUtil.RemoveNonAsciiCharacters(word);
                    if (word1.Trim().Length == 0)
                        continue;
                    if (word1 != word)
                        continue;
                    if(word1.Length<=2)
                        continue;
                    if(this._Dictionary !=null && this._Dictionary.Words !=null)
                    {
                        if (!this._Dictionary.Words.Contains(word))
                            continue;
                    }
                    if (!this._StopWords.Contains(word))
                    {
                        if (termFreqs.ContainsKey(word))
                            termFreqs[word] = termFreqs[word] + 1;
                        else
                            termFreqs.Add(word, 1);
                    }
                }
                
                foreach (string term in termFreqs.Keys)
                {
                    TermFreq termFreq = new TermFreq();
                    termFreq.Term = term;
                    termFreq.Freq = termFreqs[term];
                    termFreqs1.Add(termFreq);
                }
            }
            catch(Exception ex)
            {
                ExceptionHandler.HandlesException(ex); ;
            }
            
            return termFreqs1;
        }

        private Dictionary<string,double> CalculateTfidfScore(Dictionary<string,List<TermFreq>> termFreqsByDocs)
        {
            Dictionary<string,double> termWeights = new Dictionary<string, double>();
            Dictionary<string,int> docFrequencies=new Dictionary<string, int>();
            Dictionary<string,int> termFrequencies=new Dictionary<string, int>();
            foreach(string docName in termFreqsByDocs.Keys)
            {
                foreach(TermFreq termFreq in termFreqsByDocs[docName])
                {
                    string term = termFreq.Term;
                    if (termFrequencies.ContainsKey(term))
                        termFrequencies[term] = termFrequencies[term] + termFreq.Freq;
                    else 
                        termFrequencies.Add(term, termFreq.Freq);
                    if (docFrequencies.ContainsKey(term))
                        docFrequencies[term] = docFrequencies[term] + 1;
                    else 
                        docFrequencies.Add(term,1);
                }
            }
            foreach(string term in termFrequencies.Keys)
            {
                double weight = ((double) termFrequencies[term])/docFrequencies[term];
                termWeights.Add(term, weight);
            }
            return termWeights;
        }
        #endregion

        #region Implementation of IDisposable
        public void Dispose()
        {
            this._Dictionary.Dispose();
        }

        #endregion
    }
}
